Out of balance

imbalance
analysis
Author

Jakob Prossliner

Published

November 25, 2023

Code
library(tidyverse)
library(ospowertrader)
library(lubridate)
library(timetk)
library(arrow)

Picasso participation of Terna

Picasso data

From the transnetbw.de website we can download price and volume data for the PICASSO platform. The problem working with Picasso data is the very granular nature of the datasets, prices and volumes are published in 4 seconds intervals. In order to work with the data it is necessary to aggregate to quarthourly granularity (to see what approach is used for aggregation see the code section).

Code
prices <- arrow::open_dataset("./data/picasso/prices") %>% 
  select(dt, terna_pos, terna_neg, apg_pos, apg_neg, qo)

volumes <- arrow::open_dataset("./data/picasso/volumes") %>% 
  select(dt, terna, apg, qo)


data <- prices %>% 
  left_join(volumes, by = c("dt", "qo")) %>% 
  collect()


# i made a mistake while downloading 
# i should have parsed the single files to convert to numeric
# now i have to do it manually

## raw picasso: splice volumes in order to avoid netting of volumes (negative and positive)
picasso_df <- data %>% 
  mutate(
    terna_pos = as.numeric(terna_pos), 
    terna_neg = as.numeric(terna_neg), 
    terna = as.numeric(terna),
    apg = as.numeric(apg),
    apg_pos = as.numeric(apg_pos),
    apg_neg = as.numeric(apg_neg)
    ) %>% 
  mutate(
    terna_pos_vol = if_else(terna > 0, terna, NA),
    terna_neg_vol = if_else(terna < 0, terna, NA),
    apg_pos_vol = if_else(apg > 0, apg, NA),
    apg_neg_vol = if_else(apg < 0, apg, NA)
  )

## calculate volume weighted prices


picasso_df_aggr <- picasso_df %>% 
  group_by(qo) %>% 
  summarise(
    terna_vw_price_neg = sum(terna_pos_vol * terna_neg, na.rm = TRUE)/sum(terna_pos_vol, na.rm = TRUE),
    terna_vw_price_pos = sum(terna_neg_vol * terna_pos, na.rm = TRUE)/sum(terna_neg_vol, na.rm = TRUE),
    apg_vw_price_pos   = sum(apg_pos_vol * terna_neg, na.rm = TRUE)/sum(apg_pos_vol, na.rm = TRUE),
    apg_vw_price_neg   = sum(apg_neg_vol * terna_pos, na.rm = TRUE)/sum(apg_neg_vol, na.rm = TRUE),
    ## calculate total volume called for both zones
    terna_total_pos_called = sum(terna_pos, na.rm  = TRUE) / n(),
    terna_total_neg_called = sum(terna_neg, na.rm = TRUE) / n(),
    apg_total_pos_called   = sum(apg_pos, na.rm = TRUE) / n(),
    apg_total_neg_called   = sum(apg_neg, na.rm = TRUE) / n()
  ) %>% 
  ungroup() %>% 
  left_join(combined_prices, by = c("qo" = "df"))
Code
# picasso_df_aggr - complete dataset - picasso and imbalanceprices
# first let's analyse terna and establish whether picasso prices influence imbalance prices

# prepare terna data

terna <- picasso_df_aggr %>% select(qo, starts_with("terna"))

terna_imbalance_volues <- arrow::read_parquet(file = "./data/terna_imbalance_volumes.parquet") %>% 
  select(reference_date, imbalance = zonal_aggregate_unbalance_MWh)

terna_splits_long_short <- terna %>% 
  left_join(terna_imbalance_volues, by = c("qo" = "reference_date")) %>% 
  mutate(imbalance_bin = if_else(imbalance > 0, "long", "short")) %>% 
  split(.$imbalance_bin)

Terna

Prices

First I investigate the relationship between realized Picasso prices and imbalance prices for each imbalance sign. In the following plot we look at quarter hours with a positive control area imbalance. Indeed there is a strong link between Picasso prices and imbalance prices in the control area.

Code
# we can see indeed the relationship of volume weighted picasso prices
# and imbalance prices

terna_splits_long_short$long %>% 
  ggplot(aes(terna_price, terna_vw_price_neg))+
  geom_point(alpha = 0.3) +
  theme_bw() +
  ggtitle("Volume weighted Picasso prices vs Imbalance Prices (Postive imbalance)")

Even looking at the less extreme values we see a strong correlation between Picasso prices and imbalance prices.

Code
terna_splits_long_short$long %>% 
  filter(terna_price > -100) %>% 
  filter(terna_vw_price_neg > -100) %>% 
  ggplot(aes(terna_price, terna_vw_price_neg))+
  geom_point(alpha = 0.1) +
  geom_smooth(method = "lm")+
  theme_bw() +
  ggtitle("Volume weighted Picasso prices vs Imbalance Prices (zoomed in)")

The same strong link is visible filtering for hours with negative control area imbalance.

Code
# the same as above
terna_splits_long_short$short %>% 
  ggplot(aes(terna_price, terna_vw_price_pos))+
  geom_point(alpha = 0.3)+
  theme_bw()+
  ggtitle("Volume weighted Picasso prices vs Imbalance Prices (negative imbalance)")

Code
terna_splits_long_short$short %>% 
  filter(terna_price < 400 & terna_vw_price_pos < 500) %>% 
  ggplot(aes(terna_price, terna_vw_price_pos))+
  geom_point(alpha = 0.1)+
  geom_smooth(method = "lm")+
  theme_bw() +
  ggtitle("Volume weighted Picasso prices vs Imbalance Prices (zoomed in)")

The relationship between volume weighted prices and imbalance prices is more complex in short hours. While there exists a relationship for some quarter hours, in others the imbalance price is completely detached, a Picasso price of 0 leads to a non zero imbalance price in the control area (all the points concentrating on the 0 value of the x axis).

Volumes

We can observe that the price for secondary reserve on the Picasso platform is highly dependent on the volume requested.

Code
terna_splits_long_short$long %>% 
  ggplot(aes(terna_vw_price_neg, terna_total_neg_called))+
  geom_point(alpha = 0.4)+
  theme_bw()+
  ggtitle("Total negative sec. reserve called vs price for sec. reserve")

Let’s zoom in to see the concentration of points in the top right corner.

Code
terna_splits_long_short$long %>%
  filter(terna_total_neg_called >= -300 & terna_vw_price_neg >= -300) %>% 
  ggplot(aes(terna_vw_price_neg, terna_total_neg_called))+
  geom_point(alpha = 0.2)+
  theme_bw()+
  ggtitle("Total negative sec. reserve called vs price for sec. reserve")

Code
terna_splits_long_short$short %>% 
  ggplot(aes(terna_vw_price_pos, terna_total_pos_called))+
  geom_point(alpha = 0.4)+
  theme_bw()+
  ggtitle("Total positive sec. reserve called vs price for sec. reserve")

The same strong link between called volumes and prices is true for upwards reserve.

Code
terna_splits_long_short$short %>%
  filter(terna_vw_price_pos < 1000) %>% 
  filter(terna_total_pos_called < 1000) %>% 
  ggplot(aes(terna_vw_price_pos, terna_total_pos_called))+
  geom_point(alpha = 0.4)+
  theme_bw()+
  ggtitle("Total positive sec. reserve called vs price for sec. reserve")

Like in the case of negative aFRR called also in case of calling positive aFRR there is almost a linear relationship between volumes and prices, with the exception of quite some quarter hours, that have a price of 0 while Terna is calling volumes from the Picasso platform.

APG

Prices

Code
# prepare terna data

apg <- picasso_df_aggr %>% select(qo, starts_with("apg"))

apg_imbalance_volumes <- arrow::read_parquet(file = "./data/apg_control_area_imbalance.parquet") %>% 
  select(df, imbalance = cai)

apg_splits_long_short <- apg %>% 
  left_join(apg_imbalance_volumes, by = c("qo" = "df")) %>% 
  mutate(imbalance_bin = if_else(imbalance > 0, "long", "short")) %>% 
  split(.$imbalance_bin)
Code
apg_splits_long_short$long %>% 
  ggplot(aes(apg_price, apg_vw_price_neg))+
  geom_point(alpha = 0.3) +
  theme_bw() +
  ggtitle("Volume weighted Picasso prices vs Imbalance Prices (Postive imbalance)")

Code
apg_splits_long_short$long %>% 
  filter(apg_price > -400 & apg_vw_price_neg < 700) %>% 
  ggplot(aes(apg_price, apg_vw_price_neg))+
  geom_point(alpha = 0.3) +
  theme_bw() +
  ggtitle("Volume weighted Picasso prices vs Imbalance Prices (Postive imbalance)")

Code
apg_splits_long_short$short %>% 
  ggplot(aes(apg_price, apg_vw_price_pos))+
  geom_point(alpha = 0.3)+
  theme_bw()+
  ggtitle("Volume weighted Picasso prices vs Imbalance Prices (negative imbalance)")

Code
apg_splits_long_short$short %>% 
  filter(apg_price < 400 & apg_price > -100) %>% 
  filter(apg_vw_price_pos < 400 & apg_vw_price_pos > 0) %>% 
  ggplot(aes(apg_price, apg_vw_price_pos))+
  geom_point(alpha = 0.3)+
  theme_bw()+
  ggtitle("Volume weighted Picasso prices vs Imbalance Prices (negative imbalance)")

Overall the link between imbalance prices and picasso prices/volumes seems to be much much clearer in the case fo Terna. APG doesn’t present a strong and well identifiable link.

Volumes

Code
apg_splits_long_short$long %>% 
  ggplot(aes(apg_vw_price_neg, apg_total_neg_called))+
  geom_point(alpha = 0.4)+
  theme_bw()+
  ggtitle("Total negative sec. reserve called vs price for sec. reserve")